{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "a = torch.tensor([1,2])\n",
    "a.cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## INFERENCE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_utils.datasets import *\n",
    "import cv2\n",
    "import numpy as np\n",
    "from models import Darknet\n",
    "import torch\n",
    "from torchvision import transforms as transforms\n",
    "import torchvision\n",
    "\n",
    "img_size = 512\n",
    "file_name = '/home/lrh/program/git/object_detection/code/yolov3-train/asserts/img_17247.jpg'\n",
    "img0 = cv2.imread(file_name)\n",
    "img = letterbox(img0, new_shape=img_size)[0]\n",
    "#img = img / 255.0\n",
    "print(img.shape)\n",
    "img = img[:, :, ::-1]  # BGR to RGB, to 3x416x416\n",
    "#img = np.ascontiguousarray(img)\n",
    "img = transforms.ToTensor()(np.array(img))\n",
    "img = img.unsqueeze(0).cuda()\n",
    "\n",
    "\n",
    "model = Darknet('../trained_models/baseline-stitcher/yolov3-spp-44.cfg', img_size=img_size)\n",
    "model.load_state_dict(torch.load('../trained_models/baseline-stitcher/best.pt')['model'])\n",
    "model = model.cuda()\n",
    "model.eval()\n",
    "\n",
    "torchvision.utils.save_image(img, 'input.jpg')\n",
    "\n",
    "img_test = torch.zeros((1, 3, img_size, img_size)).cuda()\n",
    "model(img_test)\n",
    "\n",
    "pred = model(img)[0]\n",
    "\n",
    "print(pred[0,:4,:4])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## VAL DATASET\n",
    "\n",
    "command: python3 test.py --cfg trained_models/yolov3-spp-default/yolov3-spp-44.cfg --data data/rubbish.data --weights trained_models/yolov3-spp-default/best.pt --batch-size 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 上述计算AP_s, AP_m, AP_l可能有问题，下面重新尝试计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. model outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using CUDA device0 _CudaDeviceProperties(name='GeForce GTX 1080 Ti', total_memory=11175MB)\n",
      "\n",
      "Model Summary: 225 layers, 6.28049e+07 parameters, 6.28049e+07 gradients\n",
      "Fusing layers...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Caching labels (1008 found, 0 missing, 0 empty, 0 duplicate, for 1497 images):  67%|██████▋   | 1008/1497 [00:00<00:00, 10071.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model Summary: 152 layers, 6.27781e+07 parameters, 6.27781e+07 gradients\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Caching labels (1497 found, 0 missing, 0 empty, 0 duplicate, for 1497 images): 100%|██████████| 1497/1497 [00:00<00:00, 9455.61it/s] \n",
      "               Class    Images   Targets         P         R   mAP@0.5        F1: 100%|██████████| 1497/1497 [00:33<00:00, 44.87it/s]\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "import json\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from models import *\n",
    "from my_utils.datasets import *\n",
    "from my_utils.utils import *\n",
    "\n",
    "## python3 test.py --cfg yolov3-44.cfg --data data/rubbish.data --weights trained_models/yolo-v3-60e/best.pt --batch-size 8\n",
    "\n",
    "def test(cfg,\n",
    "         data,\n",
    "         weights=None,\n",
    "         batch_size=1,\n",
    "         imgsz=512,\n",
    "         conf_thres=0.001,\n",
    "         iou_thres=0.6,  # for nms\n",
    "         save_json=False,\n",
    "         single_cls=False,\n",
    "         augment=False,\n",
    "         model=None,\n",
    "         dataloader=None):\n",
    "    # Initialize/load model and set device\n",
    "    if model is None:\n",
    "        device = torch_utils.select_device('cuda', batch_size=batch_size)\n",
    "        verbose = True\n",
    "\n",
    "        # Remove previous\n",
    "        for f in glob.glob('test_batch*.jpg'):\n",
    "            os.remove(f)\n",
    "\n",
    "        # Initialize model\n",
    "        model = Darknet(cfg, imgsz)\n",
    "\n",
    "        # Load weights\n",
    "        attempt_download(weights)\n",
    "        if weights.endswith('.pt'):  # pytorch format\n",
    "            model.load_state_dict(torch.load(weights, map_location=device)['model'])\n",
    "        elif weights.endswith('.pkl'):\n",
    "            model.load_state_dict(torch.load(weights, map_location=device))\n",
    "        else:  # darknet format\n",
    "            load_darknet_weights(model, weights)\n",
    "\n",
    "        # Fuse\n",
    "        model.fuse()\n",
    "        model.to(device)\n",
    "\n",
    "        if device.type != 'cpu' and torch.cuda.device_count() > 1:\n",
    "            model = nn.DataParallel(model)\n",
    "    else:  # called by train.py\n",
    "        device = next(model.parameters()).device  # get model device\n",
    "        verbose = False\n",
    "\n",
    "    # Configure run\n",
    "    data = parse_data_cfg(data)\n",
    "    nc = 1 if single_cls else int(data['classes'])  # number of classes\n",
    "    path = data['valid']  # path to test images\n",
    "    names = load_classes(data['names'])  # class names\n",
    "    iouv = torch.linspace(0.5, 0.95, 10).to(device)  # iou vector for mAP@0.5:0.95\n",
    "    iouv = iouv[0].view(1)  # comment for mAP@0.5:0.95\n",
    "    niou = iouv.numel()\n",
    "\n",
    "    # Dataloader\n",
    "    if dataloader is None:\n",
    "        dataset = LoadImagesAndLabels(path, imgsz, batch_size, rect=True, single_cls=False)\n",
    "        batch_size = min(batch_size, len(dataset))\n",
    "        dataloader = DataLoader(dataset,\n",
    "                                batch_size=batch_size,\n",
    "                                num_workers=min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8]),\n",
    "                                pin_memory=True,\n",
    "                                collate_fn=dataset.collate_fn)\n",
    "\n",
    "    seen = 0\n",
    "    model.eval()\n",
    "    _ = model(torch.zeros((1, 3, imgsz, imgsz), device=device)) if device.type != 'cpu' else None  # run once\n",
    "    s = ('%20s' + '%10s' * 6) % ('Class', 'Images', 'Targets', 'P', 'R', 'mAP@0.5', 'F1')\n",
    "    p, r, f1, mp, mr, map, mf1, t0, t1 = 0., 0., 0., 0., 0., 0., 0., 0., 0.\n",
    "    loss = torch.zeros(3, device=device)\n",
    "    jdict, stats, ap, ap_class = [], [], [], []\n",
    "    \n",
    "    outputs = []\n",
    "    \n",
    "    for batch_i, (imgs, targets, paths, shapes) in enumerate(tqdm(dataloader, desc=s)):\n",
    "        \n",
    "        imgs = imgs.to(device).float() / 255.0  # uint8 to float32, 0 - 255 to 0.0 - 1.0\n",
    "        targets = targets.to(device)\n",
    "        nb, _, height, width = imgs.shape  # batch size, channels, height, width\n",
    "        whwh = torch.Tensor([width, height, width, height]).to(device)\n",
    "\n",
    "        # Disable gradients\n",
    "        with torch.no_grad():\n",
    "            # Run model\n",
    "            t = torch_utils.time_synchronized()\n",
    "            inf_out = model(imgs, augment=augment)[0]  # inference and training outputs\n",
    "            t0 += torch_utils.time_synchronized() - t\n",
    "            # Run NMS\n",
    "            t = torch_utils.time_synchronized()\n",
    "            ## output:\n",
    "            ### list data for each images: (nx6 (x1, y1, x2, y2, conf, cls))\n",
    "            _, output = non_max_suppression_test(inf_out, conf_thres=conf_thres, iou_thres=iou_thres)  # nms\n",
    "#             print(output[0].shape)\n",
    "#             break\n",
    "            t1 += torch_utils.time_synchronized() - t\n",
    "            \"\"\"\n",
    "            output with type(list) and shape:\n",
    "                n * 6 (xyxy， confidence, pcls)\n",
    "            \"\"\"\n",
    "            \n",
    "            # [{\"image_id\": 42, \"category_id\": 18, \"bbox\": [258.15, 41.29, 348.26, 243.78], \"score\": 0.236}, ...\n",
    "            for si, pred in enumerate(output):\n",
    "                ## 对于每张图片\n",
    "                pred = pred.clone()\n",
    "                temp = {\n",
    "                    'pred': pred[:, :4].tolist(), ## 输出bbox\n",
    "                    'conf': pred[:,4].tolist(),## object confidence.\n",
    "                    'preid': pred[:,5].tolist(), ## 输出类别\n",
    "                    'conf_all': pred[:,6:].tolist(), ##概率值\n",
    "                    'whwh': whwh.clone().tolist(), ## 输入图像大小\n",
    "                    'target': targets[targets[:, 0] == si,1:].clone().tolist(), ##目标gt [cls, xyhw]\n",
    "                    'path': paths[si],\n",
    "                    'shapes': shapes[si] #原始图像大小\n",
    "                }\n",
    "                \n",
    "                outputs.append(temp)\n",
    "                \n",
    "    return outputs\n",
    "\n",
    "cfg = '../trained_models/baseline-stitcher/yolov3-spp-44.cfg'\n",
    "data = '../data/rubbish.data' \n",
    "weights = '../trained_models/baseline-stitcher/best.pt'\n",
    "#data/rubbish.data --weights trained_models/yolov3-spp-default/best.pt\n",
    "batch_size = 1\n",
    "#data/rubbish.data --weights trained_models/yolov3-spp-default/best.pt\n",
    "\n",
    "outputs = test(cfg, data, weights, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. save & load output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(outputs))\n",
    "# for key in outputs[0].keys():\n",
    "#     print(type(outputs[0][key]))\n",
    "with open('stitcher.output.json', 'w') as f:\n",
    "    json.dump(outputs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['一次性快餐盒', '书籍纸张', '充电宝', '剩饭剩菜', '包', '垃圾桶', '塑料器皿', '塑料玩具', '塑料衣架', '大骨头', '干电池', '快递纸袋', '插头电线', '旧衣服', '易拉罐', '枕头', '果皮果肉', '毛绒玩具', '污损塑料', '污损用纸', '洗护用品', '烟蒂', '牙签', '玻璃器皿', '砧板', '筷子', '纸盒纸箱', '花盆', '茶叶渣', '菜帮菜叶', '蛋壳', '调料瓶', '软膏', '过期药物', '酒瓶', '金属厨具', '金属器皿', '金属食品罐', '锅', '陶瓷器皿', '鞋', '食用油桶', '饮料瓶', '鱼骨']\n",
      "1497\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "import json\n",
    "\n",
    "## 读取类标\n",
    "from my_utils.utils import *\n",
    "class_names = load_classes('/store/dataset/rubbish_yolo/classes.names')\n",
    "print(class_names)\n",
    "\n",
    "with open('stitcher.output.json','r') as f:\n",
    "    outputs = json.load(f)\n",
    "    \n",
    "print(len(outputs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. 计算AP small, medium, large, all."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from my_utils.utils import *\n",
    "# Statistics per image\n",
    "#obj_size_label \n",
    "## 0: SMALL, 1: MEDIUM, 2: LARGE, other: ALL\n",
    "#obj_size_label = 'medium'\n",
    "\n",
    "iouv = torch.linspace(0.5, 0.95, 10) # iou vector for mAP@0.5:0.95\n",
    "iouv = iouv[0].view(1)  # comment for mAP@0.5:0.95\n",
    "niou = iouv.numel()\n",
    "\n",
    "\n",
    "def select_with_size(*args, size_label='small', input_format='xyxy'):\n",
    "    \n",
    "    targets = args[0]\n",
    "    \n",
    "    if input_format == 'xywh':\n",
    "        target_size = targets[:, 3] * targets[:, 2]\n",
    "    else:\n",
    "        target_size = (targets[:, 3] - targets[:, 1]) * (targets[:, 2] - targets[:, 0])\n",
    "    \n",
    "    if size_label == 'small':\n",
    "        target_size_ind = np.where(target_size < 0.1)\n",
    "    elif size_label == 'medium':\n",
    "        target_size_ind = np.where((target_size> 0.1) & (target_size <0.3))\n",
    "    elif size_label == 'large':\n",
    "        target_size_ind = np.where(target_size> 0.3)\n",
    "    else:\n",
    "        ##所有样本\n",
    "        target_size_ind = np.where(target_size>=0)\n",
    "    \n",
    "    return [arg[target_size_ind] for arg in args]\n",
    "\n",
    "def calculate_AP_size(outputs, obj_size_label):\n",
    "\n",
    "    stats = []\n",
    "    paths = []\n",
    "    ## 对于每一个图像数据\n",
    "    for si, output in enumerate(outputs):\n",
    "        ## 准备数据 targets [cls_label, xywh]\n",
    "        targets = torch.tensor(output['target'])\n",
    "        ## pred: xyxy\n",
    "        pred = torch.tensor(output['pred'])\n",
    "        conf = torch.tensor(output['conf'])\n",
    "        pred_cls = torch.tensor(output['preid'])\n",
    "        whwh = torch.tensor(output['whwh'])\n",
    "        path = output['path']\n",
    "\n",
    "\n",
    "        ## 筛选\n",
    "        ### tboxs = targets[:, 1:5] tlables = targets[:, 0]\n",
    "        tboxs, tlabels = select_with_size(targets[:, 1:5], targets[:, 0], size_label = obj_size_label, input_format='xywh')\n",
    "        pboxs, pconfs, plabels = select_with_size(pred/whwh, conf, pred_cls, size_label = obj_size_label, input_format='xyxy')\n",
    "\n",
    "        pboxs = pboxs * whwh\n",
    "\n",
    "        #print(pboxs.shape)\n",
    "        ## 计算图像i中obj个数\n",
    "        nl = len(tlabels)\n",
    "        ## 将target类标转成list格式\n",
    "        #tcls = labels[:, 0].tolist() if nl else []  # target class\n",
    "\n",
    "        # Clip boxes to image bounds\n",
    "        # 将输出 clip 到图像大小\n",
    "        #print(pboxs[0])\n",
    "        clip_coords(pboxs, whwh[1:3])\n",
    "        #print(pboxs[0])\n",
    "\n",
    "        # Assign all predictions as incorrect\n",
    "        ## correct: [pred.shape[0], niou] => [[False]] \n",
    "        correct = torch.zeros(pboxs.shape[0], niou, dtype=torch.bool)\n",
    "        ## 如果有框\n",
    "        if nl:\n",
    "            detected = []  # target indices\n",
    "            #tcls_tensor = tlabels[:, 0]\n",
    "\n",
    "            # target boxes\n",
    "            ## 目标框xyxy × whwh\n",
    "            tbox = xywh2xyxy(tboxs) * whwh\n",
    "\n",
    "            # Per target class\n",
    "            for cls in torch.unique(tlabels):\n",
    "                \n",
    "#                 if(cls==7):\n",
    "#                     print(tlabels)\n",
    "#                     print(tboxs)\n",
    "                \n",
    "                ## 目标target=>类标为cls的位置\n",
    "                ti = (cls == tlabels).nonzero().view(-1)  # prediction indices\n",
    "                ## 预测pred=>类标为cls的位置\n",
    "                pi = (cls == plabels).nonzero().view(-1)  # target indices\n",
    "\n",
    "                # Search for detections\n",
    "                if pi.shape[0]:\n",
    "                    # Prediction to target ious\n",
    "                    ## 计算框之间的 iou\n",
    "                    ## shape if pred:torch.Size([9, 6]), \n",
    "                    ## shape of target:torch.Size([1, 4]), \n",
    "                    ## shape of ious:torch.Size([9]) => 表示预测框与target之间的最大交集区域的iou\n",
    "                    ## shape of i: torch.Size([9]) => 表示匹配的目标框位置。\n",
    "                    ious, i = box_iou(pboxs[pi], tbox[ti]).max(1)  # best ious, indices\n",
    "    #                         print(\"shape if pred:{}, shape of target:{}, shape of ious:{}\".format(\n",
    "    #                             pred[pi].shape, tbox[ti].shape, ious.shape))\n",
    "                    #print(\"value in ious:\", ious)\n",
    "                    # Append detections\n",
    "                    ## 对于所有 与 gt 之间iou 大于0.5的框 \n",
    "                    ## iouv = [0.5]\n",
    "                    for j in (ious > iouv[0]).nonzero():\n",
    "                        ## j 表示有效框\n",
    "                        d = ti[i[j]]  # detected target\n",
    "                        ## 对于所有目标框, 找到\n",
    "                        if d not in detected:\n",
    "                            detected.append(d)\n",
    "                            ## 如果预测的框 pi[j] 与 ti[i[j]] 交集 iou 大于iouv 则认为检测正确\n",
    "\n",
    "                            correct[pi[j]] = ious[j] > iouv  # iou_thres is 1xn\n",
    "                            if len(detected) == nl:  # all targets already located in image\n",
    "                                break\n",
    "\n",
    "        # Append statistics (correct, conf, pcls, tcls)\n",
    "        ### correct: [pred_shape, iouv] 在指定 iou threshod 下，检测到的框是否是正确的\n",
    "        stats.append((correct.cpu(), pconfs.cpu(), plabels.cpu(), tlabels.cpu()))\n",
    "        #paths.append(path)\n",
    "\n",
    "    # Compute statistics\n",
    "    stats = [np.concatenate(x, 0) for x in zip(*stats)]  # to numpy\n",
    "    return stats\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## yolo-spp\n",
    "\n",
    "small: 0.30785361123913974\n",
    "\n",
    "medium: 0.5442876200412141\n",
    "\n",
    "large: 0.792296026933334\n",
    "\n",
    "all: 0.7025410264942612\n",
    "\n",
    "## yolo-spp-stitcher\n",
    "\n",
    "small: 0.40560762181863386\n",
    "\n",
    "medium: 0.6365985867921662\n",
    "\n",
    "large: 0.8118037199254308\n",
    "\n",
    "all: 0.7263963434704765"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_levels = ['small', 'medium', 'large', 'all']\n",
    "mAPs = np.zeros([len(class_names), 0])\n",
    "\n",
    "for size_level in size_levels:\n",
    "    ## Compute mAP.\n",
    "    stats = calculate_AP_size(outputs = outputs, obj_size_label = size_level)\n",
    "    #print(stats[3])\n",
    "    from my_utils.utils import *\n",
    "    niou = 1\n",
    "    nc = 44  # number of classes\n",
    "    if len(stats):\n",
    "        p, r, ap, f1, ap_class = ap_per_class(*stats)\n",
    "        if niou > 1:\n",
    "            p, r, ap, f1 = p[:, 0], r[:, 0], ap.mean(1), ap[:, 0]  # [P, R, AP@0.5:0.95, AP@0.5]\n",
    "        mp, mr, map, mf1 = p.mean(), r.mean(), ap.mean(), f1.mean()\n",
    "        nt = np.bincount(stats[3].astype(np.int64), minlength=nc)  # number of targets per class\n",
    "    else:\n",
    "        nt = torch.zeros(1)\n",
    "    \n",
    "    ap = ap.squeeze()\n",
    "    \n",
    "    print(map)\n",
    "    \n",
    "    if size_level == 'small':\n",
    "        ap = np.insert(ap, 4, np.nan)\n",
    "        ap = np.insert(ap, 8, np.nan)\n",
    "#         ap = np.insert(ap, 9, np.nan)\n",
    "#         ap = np.insert(ap, 15, np.nan)\n",
    "#         ap = np.insert(ap, 34, np.nan)\n",
    "    elif size_level == 'medium':\n",
    "        ap = np.insert(ap, 13, np.nan)\n",
    "#         ap = np.insert(ap, 4, np.nan)\n",
    "#         ap = np.insert(ap, 22, np.nan)\n",
    "    #print(ap.squeeze().shape)\n",
    "    #print(\"mp, mr, map, mf1\", mp, mr, map, mf1)\n",
    "    \n",
    "    mAPs = np.concatenate((mAPs, ap[:,np.newaxis]), axis=1)\n",
    "\n",
    "#     for i in range(ap.shape[0]):\n",
    "#         print(ap[i])\n",
    "print(mAPs.shape)\n",
    "\n",
    "    #print(ap[ap!=0].mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = np.argsort(mAPs[:, -1])\n",
    "mAPs[index]\n",
    "np.array(class_names)[index]\n",
    "x = np.arange(len(class_names))\n",
    "print(x)\n",
    "from matplotlib import pyplot as plt\n",
    "print(mAPs[index][:,-1])\n",
    "plt.figure(figsize=(6,10))\n",
    "plt.barh(x, mAPs[index][:,-1])\n",
    "plt.yticks(x, np.array(class_names)[index])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. 统计不同类别不同size大小的个数（验证集）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "targets = np.zeros([0,5])\n",
    "for si, output in enumerate(outputs):\n",
    "    target = np.array(output['target'])\n",
    "    targets = np.concatenate((targets, target), axis=0)\n",
    "\n",
    "size_levels = ['small', 'medium', 'large', 'all']\n",
    "## 44 * 3\n",
    "size_nums = np.zeros([len(class_names), len(size_levels)])\n",
    "## cls, xywh\n",
    "#print(targets[0])\n",
    "\n",
    "for i, class_name in enumerate(class_names):\n",
    "    for sind, size_level in enumerate(size_levels):\n",
    "        target = select_with_size(targets[targets[:, 0] == i,1:], size_label=size_level, input_format='xywh')[0]\n",
    "        size_nums[i, sind] = target.shape[0]\n",
    "#print(size_nums)\n",
    "\n",
    "# targets_temp = targets[targets[:, 0] == 7,1:]\n",
    "\n",
    "results = np.concatenate((size_nums, mAPs), axis=1)\n",
    "\n",
    "for index, result in enumerate(results):\n",
    "    print(index,'-',class_names[index].ljust(9 - len(class_names[index])), ':', result)\n",
    "    #print(len(class_names[index]))\n",
    "# print(targets_temp)\n",
    "\n",
    "# target = select_with_size(targets_temp, size_label='small', input_format='xywh')[0]\n",
    "# print(target)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. 查看结果比较差的类的可视化结果"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tbox: xywh\n",
    "\n",
    "pbox: xyxy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from matplotlib import pyplot as plt\n",
    "from vutils import *\n",
    "size_level = 'all'\n",
    "\n",
    "hard_class = 35\n",
    "\n",
    "num = 0\n",
    "for i, output in enumerate(outputs):\n",
    "    predict = np.array(output['pred'])\n",
    "    conf = np.array(output['conf'])\n",
    "    conf_all = np.array(output['conf_all'])\n",
    "    target = np.array(output['target'])\n",
    "    tbox = target[:, 1:]\n",
    "    tlabel = target[:, 0]\n",
    "    \n",
    "    tbox, tlabel = select_with_size(tbox, tlabel, size_label = size_level, input_format='xywh')\n",
    "    \n",
    "    path = output['path']\n",
    "\n",
    "    if hard_class in tlabel:\n",
    "        #boxes = tbox\n",
    "        #print(tbox)\n",
    "        #print(tlabel)\n",
    "        #print(path)\n",
    "        #print(output['shapes'])\n",
    "        #print(output['whwh'])\n",
    "        #print(output['pred'])\n",
    "        pboxes = np.array(output['pred'])\n",
    "        pboxes = pboxes[np.array(output['preid']) == hard_class]\n",
    "        \n",
    "        #[conf>0.2]\n",
    "        conf_cls = conf[np.array(output['preid']) == hard_class]\n",
    "        print(conf_cls)\n",
    "        \n",
    "        conf_all_cls = conf_all[np.array(output['preid']) == hard_class]\n",
    "        print(conf_all_cls.shape)\n",
    "        print(conf_all_cls[:,[0, hard_class+1]])\n",
    "        \n",
    "        pboxes = pboxes\n",
    "        #print(pboxes[0])\n",
    "        img = Image.open(path)\n",
    "        \n",
    "        boxes = pboxes / np.array(output['whwh'])\n",
    "        whwh = np.array([img.size[0], img.size[1], img.size[0], img.size[1]])\n",
    "        #boxes = xywh2xyxy(boxes) * whwh\n",
    "        boxes = boxes * whwh\n",
    "        show_box_on_image(img, boxes)\n",
    "    \n",
    "        plt.imshow(img)\n",
    "\n",
    "        plt.show()\n",
    "        num+=1\n",
    "#         if num%10==9:\n",
    "#             break;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "a = torch.tensor([4,34])\n",
    "b = a.clone()\n",
    "\n",
    "a[0] = 5\n",
    "\n",
    "print(a)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Appendix: 测试 读取数据 接口"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
